import numpy as np
import pandas as pd
import requests
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sns
import folium
from pyproj import Proj,transform
from folium.plugins import FastMarkerCluster
from folium.plugins import MarkerCluster
from sklearn.cluster import DBSCAN
dataframe = pd.read_csv('parking-citations.csv',nrows=50000)
dataframe.head(10)
Since our primary focus is the location and type of violation along with the fine amount, we can disregard information such as Marked date and time, Plate expiry date, VIN, Make, Body style etc. We would keep the fine amount to get a sense of the amount of money being collected as fine to get a better idea on financial scale.
dataframe = dataframe.drop(['Ticket number','Meter Id','Marked Time','RP State Plate','Plate Expiry Date','Make','VIN','Body Style','Color','Route','Agency'],axis=1)
# dataframe = dataframe.dropna()
dataframe.head()
As we can see, we have dates and times on a timestamp and a 24 hour format. So the next thing we do is to see the data types of each column and try to convert it into a suitable form fo our analysis.
dataframe.dtypes
Since the date in the column is in IS08601 standard and we only need the year,month and the day,we are going to parse it to remove the time part from the data.
dataframe['Issue Date'] = pd.to_datetime(dataframe['Issue Date'], dayfirst=True)
dataframe = dataframe.sort_values(by='Issue Date')
dataframe.head()
The next thing for us would be to conver the issue time from a 24 hour format and a float data type.
dataframe['Issue time'].isna().sum()
dataframe = dataframe.dropna(subset=['Issue time'])
dataframe['Issue time'] = dataframe['Issue time'].astype(int)
# dataframe = dataframe[99965:99968]
dataframe['Issue time'] = dataframe['Issue time'].apply(str)
for i,time in enumerate(dataframe['Issue time']):
# print(time)
length = len(time)
if(length==1):
time="000"+ time
elif(length==2):
time = "00" + time
elif(length==3):
time = "0"+ time
else:
time = time
#
dataframe.at[i,"Issue time"] = time
# dataframe = dataframe[dataframe['Issue time'].str.contains(':')]
dataframe['Issue time']=pd.to_datetime(dataframe['Issue time'],format ='%H%M').dt.time
dataframe.head()
We now try to bring a hypothesis into our analysis. We pick a specific time duration which is considered to be a rush hour and try to see if the count of parking citations are in any way correlated to the number of citations.
from datetime import time
msrush = time(5,0,0)
merush = time(10,0,0)
esrush = time(16,0,0)
eerush = time(21,0,0)
print(msrush)
print(merush)
print(esrush)
print(eerush)
dataframe['Rush hour'] = np.where(((dataframe['Issue time']>msrush) & (dataframe['Issue time']<merush)) | ((dataframe['Issue time']>esrush) & (dataframe['Issue time']<eerush)),"1","0")
dataframe.head()
Exploring more into the database we now try to look various types of violation code and the number of offences commited.
counts = pd.DataFrame(dataframe['Violation code'].value_counts())
counts.plot(kind='bar',figsize=(20,15),fontsize=12)
print("The top 10 Violations are:")
counts[0:10].plot(kind='bar',figsize=(20,15))
We now compare the number of rush hour violation to the number of number of non rush hour violation
RushDF = pd.DataFrame(dataframe['Rush hour'].value_counts())
RushDF.index=['No Rush','Rush']
RushDF.plot(kind='bar', title ="Rush Hour Vs No Rush",figsize=(15,8))
As we can see some of the Fine amount have NaN as the value, our next approach would be to deal with those values. There are multiple ways to deal with this. One of the option would be to remove the values with NaN out of our analysis but it might lead us to miss out on other important information. Another way to handle the day, which we are going to use is to fill the NaN with the average fine across the parking citations.
n_rows = dataframe.shape[0]
meanFine = dataframe['Fine amount'].mean()
print(meanFine)
for i in range(0,n_rows):
if np.isnan(dataframe['Fine amount'][i]) == True:
dataframe['Fine amount'][i] = meanFine
dataframe = dataframe.dropna()
dataframe.head()
dataframe['Fine amount'] = dataframe['Fine amount'].round(0).astype(int)
rushfine = dataframe[dataframe['Rush hour'] =="0"]['Fine amount'].values.sum()
nrushfine = dataframe[dataframe['Rush hour'] =="1"]['Fine amount'].values.sum()
print("Total Amount spent in Fines:", rushfine + nrushfine )
fineDF = pd.DataFrame([rushfine,nrushfine])
fineDF.index=['Rush Hour','Non Rush Hour']
fineDF.columns=['Amoount']
fineDF.plot(kind = 'barh', figsize=(15,10), title="Rush Hour V/S Non Rush Hour Fine", fontsize=14)
start2013 = dt.datetime(2013,1,1)
end2013 = dt.datetime(2013,12,31)
start2014 = dt.datetime(2014,1,1)
end2014 = dt.datetime(2014,12,31)
start2015 = dt.datetime(2015,1,1)
end2015 = dt.datetime(2015,12,31)
start2016 = dt.datetime(2016,1,1)
end2016 = dt.datetime(2016,12,31)
start2017 = dt.datetime(2017,1,1)
end2017 = dt.datetime(2017,12,31)
# print(start2015,end2015)
dataframe.dtypes
data2013 = dataframe[(dataframe['Issue Date']> start2013) & (dataframe['Issue Date']<= end2013)].shape[0]
data2014 = dataframe[(dataframe['Issue Date']> start2014) & (dataframe['Issue Date']<= end2014)].shape[0]
data2015 = dataframe[(dataframe['Issue Date']> start2015) & (dataframe['Issue Date']<= end2015)].shape[0]
data2016 = dataframe[(dataframe['Issue Date']> start2016) & (dataframe['Issue Date']<= end2016)].shape[0]
data2017 = dataframe[(dataframe['Issue Date']> start2017) & (dataframe['Issue Date']<= end2017)].shape[0]
yearDF = pd.DataFrame([data2013,data2014,data2015,data2016,data2017])
yearDF.index =(['2013','2014','2015','2016','2017'])
yearDF.columns =(['Violation Counts'])
yearDF.plot(kind= 'bar', figsize=(15,8))
If you notice the Dataframe, the Latitude and Longitude are given on a in US Feet coordinates according to the NAD_1983_StatePlane_California_V_FIPS_0405_Feet projection. If we try and convert that, every entry with value 99999.0 lies somewhere in the Pacific ocean. So to ease our job in hand, we decided to proceed with only those coordinates whose lat long are not 99999.
dataframe = dataframe[dataframe['Latitude'] != 99999.0]
dataframe.head()
# lat = dataframe['Latitude'].values
# long = dataframe['Longitude'].values
# WGS84 = Proj(init='EPSG:4326')
# inp = Proj(init='EPSG:2229',preserve_units=True)
# dataframe['Longitude'],dataframe['Latitude']= transform(inp, WGS84, lat, long)
# dataframe = dataframe[(dataframe['Latitude'] > 34.0) & (dataframe['Latitude'] < 36.0)]
dataframe.head()
LAmap = folium.Map(location=[34.0522,-118.2437],zoom_start=10)
# LAmap
pm = '+proj=lcc +lat_1=34.03333333333333 +lat_2=35.46666666666667 +lat_0=33.5 +lon_0=-118 +x_0=2000000 ' \
'+y_0=500000.0000000002 +ellps=GRS80 +datum=NAD83 +to_meter=0.3048006096012192 +no_defs'
# convert to lat/long
x_in,y_in = dataframe['Latitude'].values, dataframe['Longitude'].values
dataframe['Longitude'],dataframe['Latitude'] = transform(Proj(pm, preserve_units = True), Proj("+init=epsg:4326"), x_in,y_in)
dataframe.head()
# from geopy.geocoders import Nominatim
# geolocater = Nominatim(user_agent="la_explorer")
# nolat = dataframe[dataframe['Latitude'] ==99999.0][:10]
# for i,val in enumerate(nolat['Location']):
# val = val + ", LA, USA"
# print(val)
# location = geolocater.geocode(val+ "LA")
# print(location)
# # nolat.at[i,"Latitude"] = location.latitude
# # nolat.at[i,"Longitude"] = location.longitude
# nolat.head()
# for lng, lat, location in zip(dataframe['Longitude'], dataframe['Latitude'],dataframe['Location']):
# FastMarkerCluster(data=list(zip(dataframe['Longitude'], dataframe['Latitude']))).add_to(LAmap)
# # print(lat,lng,location)
# label = '{}'.format(location)
# label = folium.Popup(label, parse_html=True)
# folium.CircleMarker(
# [lat, lng],
# radius=5,
# popup=label,
# color='blue',
# fill=True,
# fill_color='#3186cc',
# fill_opacity=0.7).add_to(LAmap)
# LAmap
mc = MarkerCluster()
for row in dataframe.itertuples():
mc.add_child(folium.Marker(location =[row.Latitude,row.Longitude],popup = row.Location))
LAmap.add_child(mc)
LAmap